package com.javaeye.dengyin2000.fgcrawler;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;

import java.io.*;
import java.util.*;

import com.javaeye.dengyin2000.fgcrawler.utils.DUtils;
import com.javaeye.dengyin2000.fgcrawler.utils.Squirrel;

/**
 * User: deng.yin (deng.yin@gmail.com)
 * Date: 2009-9-27
 * Time: 21:01:37
 */
public class Four4399Crawler {

    private static final int[] CATEGORY_ID = {1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 14, 16, 30, 18, 49, 19, 20, 21
            , 22, 23, 24, 25, 26, 27, 28, 29};

    private static final String NARROW_STRING = "最多评论</a></td></table><table border=\"0\" cellpadding=\"3\" cellspacing=\"0\" style=\"border-collapse: collapse\" width=\"100%\" align=\"center\">";

    private static final String FROM_URL_PATTERN1 = "<a href='{%}'><img";
    private static final String FROM_URL_PATTERN2 = "<a href=\"{%}\"><img";

    private static final String TD_PATTERN = "<td {*}>{%}</td>";

    private static final String FLASH_PATTERN = "var {*}= \"{%}.swf\";";

    private static final String PIC_PATTERN = ".htm?1024\"><IMG{*} src=\"{%}\"";
    private static final String PIC_PATTERN1 = ".htm?1024'><IMG{*} src=\"{%}\"";
    private static final String PIC_PATTERN2 = ".htm?1024><IMG{*} src=\"{%}\"";
    private static final String PIC_PATTERN3 = ".htm\"><IMG{*} src=\"{%}\"";
    private static final String PIC_PATTERN4 = ".htm><IMG{*} src=\"{%}\"";
    private static final String PIC_PATTERN5 = "true\"><IMG{*} src=\"{%}\"";
    private static final String PIC_PATTERN6 = ".htm'><IMG{*} src=\"{%}\"";

    private static final String DESCRIPTION_PATTERN = ".swf\";{*}</script>{%}<p align=\"center\">";

    private static final String CATEGORY_NAME_PATTERN = "height=38><h1>[{%}小游戏]{%}</h1>";


    public static void main(String[] args) throws IOException {
        Set<String> fromUrlSet = null;
        try {
            File file = new File(args[0]);
            if (!file.exists()) {
                file.createNewFile();
            }
            List < String > list = FileUtils.readLines((file), "utf-8");

            fromUrlSet = new HashSet<String>(list.size());
            for (String line : list) {
                if (StringUtils.isBlank(line)) {
                    continue ;
                }
                String[] strings = line.split("@@@@@");
                fromUrlSet.add(strings[0]);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }

        OutputStream os = new FileOutputStream(new File(args[0]), true);
        for (int categoryId : CATEGORY_ID) {

            int pageId = 1;
            while(true) {
                Set<String> outputSet = new HashSet<String>();
                String cpageUrl = "http://www.4399.com/flash_fl/" + categoryId + "_" + pageId + ".htm";
                String htmlSource = DUtils.getHtmlSource(cpageUrl, "gbk");
                if (StringUtils.isBlank(htmlSource) || htmlSource.indexOf("404错,请访问下面的内容") > -1) {
                    break ;
                }
                int start = htmlSource.indexOf(NARROW_STRING);
                if (start > -1) {
                    start = start + NARROW_STRING.length();
                    int end = htmlSource.indexOf("</table>", start);
                    String narrowSource = htmlSource.substring(start, end);
                    List<List<String>> tdList = Squirrel.extract(TD_PATTERN, narrowSource);
                    for (List<String> list : tdList) {
                        String line = list.get(0);
                        List<List<String>> list1 = null;
                        list1 = Squirrel.extract(FROM_URL_PATTERN1, line);
                        if (list1.isEmpty()) {
                            list1 = Squirrel.extract(FROM_URL_PATTERN2, line);
                        }
                        if (!list1.isEmpty()) {
                            String from = list1.get(0).get(0);
                            if (from.startsWith("..")) {
                                from = from.substring(2);
                            }
                            from = "http://www.4399.com" + from;
                            if (fromUrlSet.contains(from)) {
                                continue ;
                            }
                            System.out.println(from);
                            StringBuilder sb = new StringBuilder();
                            sb.append(from);
                            String realSource = DUtils.getHtmlSource(from, "gbk");
                            List<List<String>> list2 = Squirrel.extract(FLASH_PATTERN, realSource);
                            if (list2.isEmpty()) {
                                continue ;
                            }
                            sb.append("@@@@@" + list2.get(0).get(0) + ".swf");
                            List<List<String>> list5 = Squirrel.extract(CATEGORY_NAME_PATTERN, realSource);
                            sb.append("@@@@@" + list5.get(0).get(0) + "@@@@@" + list5.get(0).get(1));
                            List<List<String>> list3 = Squirrel.extract(PIC_PATTERN, realSource);
                            if (list3.isEmpty()) {
                                list3 = Squirrel.extract(PIC_PATTERN1, realSource);
                            }
                            if (list3.isEmpty()) {
                                list3 = Squirrel.extract(PIC_PATTERN2, realSource);
                            }
                            if (list3.isEmpty()) {
                                list3 = Squirrel.extract(PIC_PATTERN3, realSource);
                            }
                            if (list3.isEmpty()) {
                                list3 = Squirrel.extract(PIC_PATTERN4, realSource);
                            }
                            if (list3.isEmpty()) {
                                list3 = Squirrel.extract(PIC_PATTERN5, realSource);
                            }
                            if (list3.isEmpty()) {
                                list3 = Squirrel.extract(PIC_PATTERN6, realSource);
                            }
                            sb.append("@@@@@" + list3.get(0).get(0));
                            List<List<String>> list4 = Squirrel.extract(DESCRIPTION_PATTERN, realSource);
                            String desc = list4.get(0).get(0).trim();
                            desc = desc.replaceAll("\n", "");
                            desc = desc.replaceAll("\r", "");
                            sb.append("@@@@@" + desc);
                            System.out.println(sb.toString());
                            outputSet.add(sb.toString());
                        }
                    }
                }
                System.out.println("category:" + categoryId + ", pageId:" +pageId);
                pageId++;
                IOUtils.writeLines(outputSet, null, os, "utf-8");
            }


        }

    }

}
